package com.wl.demosparkstream

import org.apache.log4j.{Level, Logger}
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}

object FileDStream {
  def main(args: Array[String]): Unit = {

    Logger.getLogger("org").setLevel(Level.ERROR)
    val conf = new SparkConf().setAppName(this.getClass.getCanonicalName).setMaster("local[*]")


    val ssc = new StreamingContext(conf, Seconds(5))

    val lines: DStream[String] = ssc.textFileStream("D:\\MyStudent\\studentCode\\DemoSpark\\Demo1\\data\\test1")

    println(lines)

    val worldsCount: DStream[(String, Int)] = lines.flatMap(_.split("\\s+")).map((_, 1)).reduceByKey(_ + _)

    worldsCount.print()

    ssc.start()
    ssc.awaitTermination()

  }
}
